
#--------- Install packages to for today's session ----------#
# install.packages("tidyverse", dependencies = TRUE) # This package should already be installed on Jupyter Hub

#--------- Load packages to for today's session ----------#
library(tidyverse)

Warning message:
"package 'tidyverse' was built under R version 4.0.5"
-- Attaching packages --------------------------------------- tidyverse 1.3.1 --

v ggplot2 3.3.3     v purrr   0.3.4
v tibble  3.1.1     v dplyr   1.0.6
v tidyr   1.1.3     v stringr 1.4.0
v readr   1.4.0     v forcats 0.5.1

Warning message:
"package 'ggplot2' was built under R version 4.0.5"
Warning message:
"package 'tibble' was built under R version 4.0.5"
Warning message:
"package 'tidyr' was built under R version 4.0.5"
Warning message:
"package 'dplyr' was built under R version 4.0.5"
Warning message:
"package 'forcats' was built under R version 4.0.5"
-- Conflicts ------------------------------------------ tidyverse_conflicts() --
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()


# Assign our sequence to dino
dino <- ">DinoDNA from Crichton JURASSIC PARK  p. 103 nt 1-1200 GCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAAAATCGACGCGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCTCCCTCGTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGAAGCGTGGCTGCTCACGCTGTACCTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGGGCTGTGTGCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACCCGGTAAAGTAGGACAGGTGCCGGCAGCGCTCTGGGTCATTTTCGGCGAGGACCGCTTTCGCTGGAGATCGGCCTGTCGCTTGCGGTATTCGGAATCTTGCACGCCCTCGCTCAAGCCTTCGTCACTCCAAACGTTTCGGCGAGAAGCAGGCCATTATCGCCGGCATGGCGGCCGACGCGCTGGGCTGGCGTTCGCGACGCGAGGCTGGATGGCCTTCCCCATTATGATTCTTCTCGCTTCCGGCGGCCCGCGTTGCAGGCCATGCTGTCCAGGCAGGTAGATGACGACCATCAGGGACAGCTTCAACGGCTCTTACCAGCCTAACTTCGATCACTGGACCGCTGATCGTCACGGCGATTTATGCCGCACATGGACGCGTTGCTGGCGTTTTTCCATAGGCTCCGCCCCCCTGACGAGCATCACAAACAAGTCAGAGGTGGCGAAACCCGACAGGACTATAAAGATACCAGGCGTTTCCCCCTGGAAGCGCTCTCCTGTTCCGACCCTGCCGCTTACCGGATACCTGTCCGCCTTTCTCCCTTCGGGCTTTCTCAATGCTCACGCTGTAGGTATCTCAGTTCGGTGTAGGTCGTTCGCTCCAAGCTGACGAACCCCCCGTTCAGCCCGACCGCTGCGCCTTATCCGGTAACTATCGTCTTGAGTCCAACACGACTTAACGGGTTGGCATGGATTGTAGGCGCCGCCCTATACCTTGTCTGCCTCCCCGCGGTGCATGGAGCCGGGCCACCTCGACCTGAATGGAAGCCGGCGGCACCTCGCTAACGGCCAAGAATTGGAGCCAATCAATTCTTGCGGAGAACTGTGAATGCGCAAACCAACCCTTGGCCATCGCGTCCGCCATCTCCAGCAGCCGCACGCGGCGCATCTCGGGCAGCGTTGGGTCCT"


# Remove the pattern that matches a ">"
str_remove(string = dino, pattern = ">")


# Remove the first instance of an integer 0-9
str_remove(string = dino, pattern = "[0-9]")


# Remove all instances of integers 0-9
str_remove_all(string = dino, pattern = "[0-9]")


# Remove all spaces from dino
str_remove_all(string = dino, pattern = "\\s")


# What can we remove without using the quantifier?
str_remove(string = dino, pattern = ">.") 

# What can we remove without using the quantifier?
str_remove(string = dino, pattern = ">.[0-9]\\s") 

# removes > and D, the first two characters in the string


# Remove the entire header from the sequence
str_remove(string = dino, pattern = ">.*[0-9]\\s") # remove entire header


# What happens if we modify our * with lazy matching?
str_remove(string = dino, pattern = ">.*?[0-9]\\s") 

# lazy matching removes the first occurrence of a group of digits followed by a space


# Save DNA sequence using the correct header Regex
dna <- str_remove(string = dino, pattern = ">.*[0-9]\\s")
dna


bad_string = "you\\stay\\here"

# How the string is stored
bad_string
# How it is interpreted 
cat(bad_string)

you\stay\here


# Remove the u\\s from our string
str_remove(bad_string, pattern = "u\\\\s")


# Use a raw string literal to remove u\\s from our bad_string
str_remove(bad_string, pattern = r"(u\\s)")


# Our original code for removing spaces in our header
str_remove_all(string = dino, pattern = "\\s")


# Convert it to a raw string literal. Will this work?
str_remove_all(string = dino, pattern = r"(\\s)")


# Use the proper raw string literal
str_remove_all(string = dino, pattern = r"(\s)")


# Save the header information to a variable
header <- str_extract(string = dino, pattern = r"(>.*[0-9])")

# Check that it worked as expected
header


# Use extract all to find any Gs or Cs
str_extract_all(dna, pattern = "G|C")

# This is equivalent when using str_extract_all
str_extract_all(dna, pattern = "[GC]")


# Take advantage of helper functions like str_length()
str_count(dna, pattern = "G|C")/str_length(dna) * 100 # percentage of GC content


# Use str_replace_all to transcribe our DNA sequence
mrna <- str_replace_all(string = dna, 
                        pattern = c("G" = "X", # placeholder to prevent all C from becoming G
                                    "C" = "G", 
                                    "A" = "U", 
                                    "T" = "A",
                                    "X" = "C"))

# Look at the resulting mRNA sequence
mrna


# Check for the presence of a start codon
str_detect(string = mrna, 
           pattern = "AUG")


str_count(mrna, "AUG")


# try out string locate on mrna
...(c(mrna, mrna), "AUG")

# vs string locate all on mrna
...(mrna, "AUG")


# Examine the mRNA sequence
mrna


# Subsetting from position 90 (beginning of start codon) to base 1,200 (last base in our sequence). 
# Note that the beginning of the sequence changed
str_sub(mrna, ... = 90, ... = 1200) 

# is equivalent to
str_sub(mrna, str_locate(mrna, "AUG")[1]) # Look at the nested code (one function inside another)


# Equivalent code using the piping format
mrna %>% 
str_locate(., "AUG") %>%  # Locate the first "AUG"
... %>%                  # Take it's location from the first column of the output
str_sub(mrna, .)          # Retrieve the mRNA sequence from that point onwards


# Replace the original mRNA sequence with the trimmed version
mrna <- str_sub(mrna, str_locate(mrna, "AUG")[1])

mrna


# think of these dots as wildcards to pick any character triplets
str_extract_all(mrna, ...) 

# how many codons did we retrieve?
length(str_extract_all(mrna, ...))


# Unlist our results
codons <- 
    str_extract_all(mrna, "...") %>%  # Make the extraction
    ...                          # Unlist the resulting extracted output

# Take a look at the resulting
codons

length(codons)


str_detect(codons, "(UAG)|(UGA)...")


# Which returns a numerical vector of indices 
...(codons, "UAG|UGA|UAA") 

# Note that this is equivalent
which(str_detect(codons, ...)) # Output is boolean


# Simply grab coodons by index position
codons[...]

#equivalent to 
translation <- codons[...] 
# The search will be stopped at the first detection of either of the stop codons listed

translation


unique(translation)

codon_translation = c("UUU" = "F", "UCU" = "S", "UAU" = "Y", "UGU" = "C", 
                      "UUC" = "F", "UCC" = "S", "UAC" = "Y", "UGC" = "C", 
                      "UUA" = "L", "UCA" = "S", "UAA" = "*", "UGA" = "*", 
                      "UUG" = "L", "UCG" = "S", "UAG" = "*", "UGG" = "W", 
                      "CUU" = "L", "CCU" = "P", "CAU" = "H", "CGU" = "R", 
                      "CUC" = "L", "CCC" = "P", "CAC" = "H", "CGC" = "R", 
                      "CUA" = "L", "CCA" = "P", "CAA" = "Q", "CGA" = "R", 
                      "CUG" = "L", "CCG" = "P", "CAG" = "Q", "CGG" = "R", 
                      "AUU" = "I", "ACU" = "T", "AAU" = "N", "AGU" = "S", 
                      "AUC" = "I", "ACC" = "T", "AAC" = "N", "AGC" = "S", 
                      "AUA" = "I", "ACA" = "T", "AAA" = "K", "AGA" = "R", 
                      "AUG" = "M", "ACG" = "T", "AAG" = "K", "AGG" = "R", 
                      "GUU" = "V", "GCU" = "A", "GAU" = "D", "GGU" = "G", 
                      "GUC" = "V", "GCC" = "A", "GAC" = "D", "GGC" = "G", 
                      "GUA" = "V", "GCA" = "A", "GAA" = "E", "GGA" = "G", 
                      "GUG" = "V", "GCG" = "A", "GAG" = "E", "GGG" = "G")


translation <- str_replace_all(translation, ...)


translation


...(translation)

# Equivalent call using the paste function
paste(translation, collapse="")

translation <- ...
translation


# recombine header with our translation and separate with a space
...(header, translation, sep = ...)


# Take a quick look at our directory
getwd()
list.files("data/")


# read file as a single string
fasta <- read_file("data/FoxP2_primate.fasta")

# How big is this string?
...

# a single continuous string with all genes. 
print(str_sub(...))

# Note the line breaks!


# Retrieve the structure 
str(fasta)

# is it a vector?
...(fasta)

# What are it's dimensions?
dim(fasta) # Is unidimensional so the output of dim() is NULL


# We'll practice our piping here again at the same time
fasta %>% 
# Subset our fasta string
str_sub(., 1, 2000) %>% 
# Pass it along to writeLines which will properly interpret the line breaks
...


# We'll practice our piping here again at the same time
fasta %>% 
# Subset our fasta string
str_sub(., 1, 2000) %>% 
# Another base-r alternative, cat will also properly interpret the line breaks
...


# Count the number of > in fasta
str_count(string = fasta, pattern = ...)


# Make a smaller version of our string to play with
subfasta <-
fasta %>% 
str_sub(., 1, 2000)


subfasta  %>% 
str_replace_all(pattern = ...,     # look for opening or closing squared brackets
                replacement = ...) %>%    # replace squared brackets with nothing
writeLines() # Let's make it easier to look at


# Use str_split to break up the line breaks
subfasta  %>% 
str_replace_all(pattern = r"(\[|\])", replacement = "") %>% # Remove square brackets
str_split(...) %>% # Use str_split to break up the line breaks. On unix you can use \\n\\n
head()


# You need to convert the result to a vector to work with.
subfasta  %>% 
str_replace_all(pattern = r"(\[|\])", replacement = "") %>% # Remove square brackets
str_split(r"(\r\n\r\n)")  %>% # Use str_split to break up the line breaks. On unix you can use \\n\\n
... # Convert our list to a character vector


# remove the ">" from the start of each entry
subfasta  %>% 
str_replace_all(pattern = r"(\[|\])", replacement = "") %>% # Remove square brackets
str_split(r"(\r\n\r\n)")  %>% # Use str_split to break up the line breaks. On unix you can use \\n\\n
unlist()  %>% # You need to unlist the result to work with easier
str_replace_all(pattern = ..., replacement = "") # remove the ">" from the start of each entry


# Break each entry into 2 columns with str_split_fixed and save it
header_seq_fasta <- fasta %>% 
str_replace_all(pattern = r"(\[|\])", replacement = "") %>% # Remove square brackets
str_split(r"(\r\n\r\n)")  %>% # Use str_split to break up the line breaks. On unix you can use \\n\\n
unlist()  %>% # You need to unlist the result to work with easier
str_replace_all(pattern = "^>", replacement = "") %>%  # remove the ">" from the start of each entry
str_split_fixed(pattern = ..., n = ...)  %>% # Split the string by the first occurence of our pattern
as.data.frame()


head(header_seq_fasta)


# Which entries within our dataset are duplicated by sequence?
which(...)


# Use which to filter out duplicated copies
header_seq_subset <- header_seq_fasta[...]

# Take a look at the resulting subset
str(header_seq_subset)


# Use filter to remove duplicated entries
header_seq_subset <- 
header_seq_fasta %>% # Pass the header/seq data to filter
filter(...) # identify the duplicated entries and filter on the negated result

str(header_seq_subset)


fasta_header <-
header_seq_subset[,1] %>% # Pass along the data frame
# take just the first column (using select will return a data frame, we want a vector)
str_match_all(...)

# Take a look at the last few rows of our result
tail(fasta_header, 3)


# Look at the first entry of fasta_header
fasta_header[[1]]

# Two ways to drop the first column
print("Select our columns specifically...")
fasta_header[[1]][...]

print("Leave a column behind")
fasta_header[[1]][...]


# Convert the results of str_match_all into a data frame
...(fasta_header[[1]][,2:5])

# You need to transpose the trimmed data for proper coercion
data.frame(...(fasta_header[[1]][, 2:5]))


str(fasta_header[[2]])


# What happens if we call rbind?
...(fasta_header[[1]][,-1], fasta_header[[2]][,-1])

# What about rbind.data.frame?
...(fasta_header[[1]][,-1], fasta_header[[2]][,-1])


# Create a data frame and place-hold the column names to match with what we have in fasta6
header.df <- rbind.data.frame(fasta_header[[1]][,-1])

rm(i) ## remove any previous instances of i

for (i in ...){

    header.df <- rbind.data.frame(header.df, fasta_header[[...]][, -1])
}

head(header.df)
str(header.df)


# Let's see what happens if we use lapply first
...(fasta_header, rbind.data.frame) %>%  head()


# Now use do.call
header.df <- do.call(rbind.data.frame, fasta_header) %>% 
select(-1) # remember you need to get rid of the first column


# rename the columns "Accession_number", "Protein_info", "Genus", and "Species"
... <- c("Accession_number", "Protein_info", "Genus", "Species")

head(header.df,2)


# Confirm what our sequence data looks like
print(header_seq_subset[...])


# Let's split off all of the extra line breaks
header_seq_subset[,2]  %>% 
str_split(...) %>% 
# Now you'll have a list of split sequences that you'll need to re-unite
lapply(., ...)  %>%  
# Turn this result back into a vector
unlist()  %>% 
# drop the last element because it's not helpful
head(-1)


# Let's remove all of the extra line breaks
seq_united <- header_seq_subset[,2]  %>% 
...(r"(\r\n)") %>% 
# Now you'll have just have a vector that you need to trim
head(-1)

# Take a peek at the result
print(head(seq_united))


# we can combine the columns, but will we get the column names we want?
# Remember we already have the sequences in header.df
head(cbind(header.df[, -5], ...))


# If we use dplyr and  %>%  we can keep passing this around for further use
header.df %>% 
...  %>%  
head()


# Replace the sequence column of our header.df but this is a permanent command!
header.df...
head(header.df)


writeLines("hello\n\U0001F30D") # U1F30D prints a planet icon on datacamp but not here
# similar to cat. Unicode characters in: http://www.unicode.org/charts/
cat("hello\n\U0001F30D")


# Use writeLines() with "\u0928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e"
# Hello world in Hindi (taken from datacamp.com)
writeLines("\U00000928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e")
cat("\u0928\u092e\u0938\u094d\u0924\u0947 \u0926\u0941\u0928\u093f\u092f\u093e")

Expression	Meaning
\w, [A-z, 0-9], [[:alnum:]]	word characters (letters + digits)
\d, [0-9], [[:digit:]]	digits
[A-z], [[:alpha:]]	alphabetical characters
\s, [[:space:]]	space
[[:punct:]]	punctuation
[[:lower:]]	lowercase
[[:upper:]]	uppercase
\W, [^A-z0-9]	not word characters
\S	not space
\D, [^0-9]	not digits

Expression	Meaning
?	0 or 1 occurrence
*	0 or more occurrences
+	1 or more occurrences
{n}	exactly n occurrences
{n,}	at least n occurrences
{,n}	at most n occurrences
{n,m}	between n and m occurrences (inclusive)

Expression	Meaning
\|	or
.	matches any single character
[ ... ]	matches ANY of the characters inside the brackets
[ ... - ... ]	matches a RANGE of characters inside the brackets
[ ^... ]	matches any character EXCEPT those inside the bracket
( ... )	grouping - used for [backreferencing] (https://www.regular-expressions.info/backref.html)

Expression	Meaning
^	start of the string
$	end of the string
\\b	empty string at either edge of the word
\\B	empty string that is NOT at the edge of a word

Expression	Meaning
\\	escape for meta-characters to be used as characters (*, $, ^, ., ?, \|, \, [, ], {, }, (, )).
	Note: the backslash is also a meta-character.

Expression	Meaning
^	Beginning of a line
>	Greater than symbol, used to designate the beginning of a new read in fasta format
\w	Match a word (alphabetic) character (Remember that a second escape character (backslash) is required)
+	Match the preceding metacharacter more than once
.	match a literal period
\d	Match a digit
\s	space
.+	anything, one or more times
\[	match a literal squared bracket
.*	anything, as many times it is present

Text Entry	Properties
XP_007980836.1	alphanumeric series, separated by ".", ending with digits
PREDICTED: forkhead box protein P2 isoform X2	alphanumeric with any number of words and spaces present
Chlorocebus	single alphabetical word
sabaeus	single alphabetical word

Lecture 05: Of Data Cleaning and Documentation - Conquer Regular Expressions and Challenge yourself with a 'Real' Dataset¶

0.1.0 About Introduction to R¶

0.1.1 Where is this course headed?¶

0.1.2 How do we get there? Step-by-step.¶

0.2.0 Class Objectives¶

0.3.0 A legend for text format in Jupyter markdown¶

0.4.0 Lecture and data files used in this course¶

0.4.1 Weekly Lecture and skeleton files¶

0.4.2 Live-coding HTML page¶

0.4.3 Post-lecture PDFs and Recordings¶

0.4.4 Data used in this session¶

0.4.4.1 Datafile 1: data/regex_word.docx¶

0.4.4.2 Datafile 2: data/FoxP2_primate.fasta¶

0.5.0 Packages used in this lesson¶

1.0.0 Data cleaning or "data munging" or "data wrangling"¶

2.0.0 Introduction to regular expressions (RegEx)¶

2.0.1 Regular expressions can be your friends¶

2.1.0 Classes are groups of reserved characters¶

2.2.0 Quantifiers denote repetition of patterns¶

2.3.0 Operators are helper actions¶

2.4.0 Matching to the start or end of a string¶

2.5.0 Metacharacters must be escaped with \¶

2.6.0 Trouble-shooting RegEx¶

3.0.0 Online ReGex exercise¶

3.1.0 Breaking down a regular expression example¶

3.1.1 The breakdown:¶

3.2.0 Introduction to string manipulation with stringr¶

3.2.1 RStudio's find-replace¶

3.2.2 Remove unwanted text with str_remove() and str_remove_all()¶

3.2.2.1 str_remove() replaces the first instance of a search string versus str_remove_all()¶

3.2.2.2 Reminder to escape your escape characters¶

3.2.2.3 Combine search terms into a single string¶

3.2.3 Greedy vs. lazy matching¶

3.3.0 Assign raw string literals with r"( )" to simplify your expressions¶

3.4.0 Extracting to save for later with str_extract()¶

3.5.0 Searching with str_extract_all()¶

3.6.0 Replacement using str_replace_all()¶

3.7.0 Useful search tools powered by RegEx¶

3.7.1 Simple questions with str_detect()¶

3.7.2 Counting pattern matches with str_count()¶

3.7.3 Locate pattern matches with str_locate()¶

3.8.0 Splitting up our string with str_sub()¶

3.9.0 Applying our RegEx skills further¶

3.9.1 Generate codons from the first open reading frame with a ...¶

3.9.1.1 Remember some functions return a list as output.¶

3.9.2 Search for multiple specific patterns using |¶

3.9.3 Identify specific occurrences in a vector with str_which()¶

3.9.4 Replace your codons with amino acids¶

3.9.5 Collapsing your results into a single string with str_flatten()¶

3.9.6 Recombining strings with str_c()¶

4.0.0 Multi FASTA file formatting¶

4.0.1 If you wanted to pull down your own version of the data set¶

4.1.0 Reorganizing a multi-entry fasta file¶

4.1.1 Read in your file as a single string using readr::read_file()¶

4.1.2 Explore your data to understand its structure¶

4.1.2.1 Use writelines() or cat() to see your output in a human-readable format¶

4.1.3 How many entries do we have?¶

4.2.0 Identify a structural pattern and come up with a plan¶

4.2.1 Strip out and replace all occurences of [ and ]¶

4.2.2 Split each entry into an individual row¶

4.2.3 Remove the > from the start of each entry¶

4.2.4 Split your entries into a header and sequence with str_split_fixed()¶

4.2.4.1 Remove duplicated() entries with the help of which()¶

4.2.4.2 Remove duplicated() entries with the help of filter()¶

4.2.5 Break up the header column into the components that we want¶

4.2.6 Reformat our header information into a data frame¶

4.2.6.1 A note about working with data frames and rbind()¶

4.2.6.2 Use a loop to build your data frame one row at a time¶

4.2.6.3 do.call() to save youself some trouble¶

4.2.7 Reformat our sequence entries with str_*() methods¶

4.2.8 Adding columns to your data frame - a tool for each purpose¶

5.0.0 Class summary¶

5.1.0 Post-lecture assessment (12% of final grade)¶

5.2.0 Acknowledgements¶

5.3.0 Your DataCamp academic subscription¶

5.4.0 Resources¶

6.0.0 Appendix: A real messy dataset challenge¶

6.1.0 Just for fun¶

2.5.0 Metacharacters must be escaped with `\`¶

3.2.0 Introduction to string manipulation with `stringr`¶

3.2.2 Remove unwanted text with `str_remove()` and `str_remove_all()`¶

3.2.2.1 `str_remove()` replaces the first instance of a search string versus `str_remove_all()`¶

3.3.0 Assign raw string literals with `r"( )"` to simplify your expressions¶

3.4.0 Extracting to save for later with `str_extract()`¶

3.5.0 Searching with `str_extract_all()`¶

3.6.0 Replacement using `str_replace_all()`¶

3.7.1 Simple questions with `str_detect()`¶

3.7.2 Counting pattern matches with `str_count()`¶

3.7.3 Locate pattern matches with `str_locate()`¶

3.8.0 Splitting up our string with `str_sub()`¶

3.9.1 Generate codons from the first open reading frame with a `...`¶

3.9.1.1 Remember some functions return a `list` as output.¶

3.9.2 Search for multiple specific patterns using `|`¶

3.9.3 Identify specific occurrences in a vector with `str_which()`¶

3.9.5 Collapsing your results into a single string with `str_flatten()`¶

3.9.6 Recombining strings with `str_c()`¶

4.1.1 Read in your file as a single string using `readr::read_file()`¶

4.1.2.1 Use `writelines()` or `cat()` to see your output in a human-readable format¶

4.2.1 Strip out and replace all occurences of `[` and `]`¶

4.2.3 Remove the `>` from the start of each entry¶

4.2.4 Split your entries into a header and sequence with `str_split_fixed()`¶

4.2.4.1 Remove `duplicated()` entries with the help of `which()`¶

4.2.4.2 Remove `duplicated()` entries with the help of `filter()`¶

4.2.6.1 A note about working with data frames and `rbind()`¶

4.2.6.3 `do.call()` to save youself some trouble¶

4.2.7 Reformat our sequence entries with `str_*()` methods¶